import requests
from fake_useragent import UserAgent
from lxml import etree
from pymongo import MongoClient

url = "https://www.17k.com/list/493239.html"
headers = {'User-Agent': UserAgent().random}
html = requests.get(url=url, headers=headers, timeout=30)
html.encoding = 'utf-8'
REPLICASET_URL = 'mongodb://10.0.0.10:27017,10.0.0.11:27017,10.0.0.12:27017/?replicaSet=GRIDSUM'
testDb = 'test'
testColl = 'testColl'
# 获取mongoclient。
client = MongoClient(REPLICASET_URL)


def get_title(html):
    p = etree.HTML(html)
    li_list = p.xpath('//div[@class="Main List"]/dl/dd/a[@target="_blank"]')

    for li in li_list:
        name = li.xpath('.//span/text()')
        print(name[0].strip())
        href = li.xpath('.//@href')
        con = get_con(url='https://www.17k.com' + href[0].strip())
        print(str(con))
        inster_mysql(title=name[0].strip(), con=con)


def get_con(url):
    html = requests.get(url=url, headers=headers, timeout=3)
    html.encoding = 'utf-8'
    info = etree.HTML(html.text)
    con = info.xpath('/html/body/div[4]/div[2]/div[2]/div[1]/div[2]/p/text()')
    return con


def inster_mysql(title, con):
    item = {title: str(con)}
    if len(con) > 0:
        doc = dict(item)
        doc_id = client.testDb.testColl.insert_one(doc).inserted_id
        print('doc_id:', doc_id)


for num in range(1000):
    get_title(html=html.text)

client.close()